{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# PyTorch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import torch\n",
    "import numpy as np\n",
    "\n",
    "torch.cuda.is_available()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[0 1 2 3]\n",
      " [4 5 6 7]]\n",
      "tensor([[0, 1, 2, 3],\n",
      "        [4, 5, 6, 7]])\n",
      "[[0 1 2 3]\n",
      " [4 5 6 7]]\n"
     ]
    }
   ],
   "source": [
    "# numpy data to torch data\n",
    "\n",
    "np_data = np.arange(8).reshape((2, 4))\n",
    "\n",
    "torch_data = torch.from_numpy(np_data)\n",
    "\n",
    "print(np_data)\n",
    "print(torch_data)\n",
    "\n",
    "# reverse\n",
    "np_data2 = torch_data.numpy()\n",
    "print(np_data2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 7 12]\n",
      " [18 31]]\n",
      "tensor([[ 7, 12],\n",
      "        [18, 31]])\n"
     ]
    }
   ],
   "source": [
    "# 矩阵相乘\n",
    "np_data = np.array([[1, 2], [3, 5]])\n",
    "torch_data = torch.from_numpy(np_data)\n",
    "print(np_data.dot(np_data))\n",
    "print(torch_data.mm(torch_data))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Variable\n",
    "Tensor是PyTorch中的基础组件，但是构建神经网络需要构计算图的Tensor，即Variable。\n",
    "\n",
    "包含以下三个属性：\n",
    "    1. data：通过`.data`访问，Tensor本身\n",
    "    2. grad：对应Tensor的梯度\n",
    "    3. creator：通过`.grad_fn`访问，可用户回溯整个创建链路"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "from torch.autograd import Variable"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[-1.2710, -1.2174,  0.7580,  1.2570, -1.9487],\n",
      "        [ 0.4240, -1.0563, -1.2588, -1.1750, -1.1221],\n",
      "        [-0.2329, -0.6273,  0.1150,  0.5023, -1.2057],\n",
      "        [-0.0072,  1.4407, -1.3471,  0.5326, -0.5957],\n",
      "        [ 0.4002, -0.7410, -1.3477,  0.2573,  2.4653],\n",
      "        [-0.7514,  0.6866, -0.2675, -0.7210,  0.3294],\n",
      "        [ 0.0439, -1.7097, -0.1005, -0.3069,  0.1006],\n",
      "        [-0.1989, -0.0553, -1.6201,  0.9935, -0.5913],\n",
      "        [-0.1965,  0.9328,  0.1255, -1.0535, -0.5692],\n",
      "        [-0.1148, -1.1736,  0.8110,  2.7152,  1.2115]])\n",
      "None\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "x_tensor = torch.randn(10, 5) # 从标准正态中返回多个值\n",
    "x = Variable(x_tensor, requires_grad=True) # 默认Variable不需要求梯度， 因此 添加此参数强制求梯度\n",
    "print(x.data)\n",
    "print(x.grad)\n",
    "print(x.grad_fn)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 激活函数\n",
    "\n",
    "PyTorch中常见的激活函数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAewAAAFlCAYAAAApldtwAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/d3fzzAAAACXBIWXMAAAsTAAALEwEAmpwYAAA7j0lEQVR4nO3dd5wU9f3H8deHjoCCCIoUQYMVFPHAbrBjAzVqNDaiESNiib3EWNDEkthbELDkh2JXVBBRmhhRilgoRkSMBxiKiAiccPL5/fFd8Djujjtud2Z29/18POaxbW7nfbs7+9n5zsz3a+6OiIiIJFuNuAOIiIjIxqlgi4iIZAEVbBERkSyggi0iIpIFVLBFRESygAq2iIhIFqgVd4DybLXVVt62bdu4Y4gk3uTJkxe5e7O4c1RE67NI5VS0Pie2YLdt25ZJkybFHUMk8czs67gzbIzWZ5HKqWh9VpO4iIhIFoi0YJtZYzN7wcxmmtkMM9s3yuWLSMXMbJCZLTCzz8p53MzsfjObZWafmFnnqDOK5Kuot7DvA950952BPYAZES9fRCr2BNC9gsePAtqnpt7AIxFkEhEi3IdtZlsABwG9ANx9FbCqKs+xevVqCgsLKSoqSn/AmNWrV49WrVpRu3btuKNIUnz0EXz7LRx1VGSLdPdxZta2gll6Ak95GIRgQqrVrIW7z6/qsnJ5fU4nfTckmDsUFcGKFbB8ebhce72oCH76CVat+uVy//3hV7/a5MVFedBZO2Ah8LiZ7QFMBi5x9+VrZzCz3oRf7bRp02aDJygsLKRRo0a0bdsWM4smdQTcncWLF1NYWEi7du3ijiNJUFQEZ5wBS5fCF19A/fpxJ1qrJfBNiduFqfs2KNj5uj6nk74bIrJmDSxYAPPmhenbb+G772DJkg0vlyyBH3/8pThXZQCtQYOypmDXAjoDF7n7B2Z2H3ANcMPaGdy9P9AfoKCgYINXoaioKCdXbjOjadOmLFy4MO4okhQ33ADTp8Pw4Ukq1lWSr+tzOum7IY1++AFmzoRZs8KP4FmzwlRYGAp0cfGGf1O7Nmy5JTRpEi5btIBdd4VGjWCzzcLUoEHZ1+vVg7p1oU6dXy6bN6/WvxBlwS4ECt39g9TtFwgFu0pydeXO1f9LNsG778I//gHnnw/dK9qdHIu5QOsSt1ul7tsk+txvnF6jTbBkCbz/ftitNHVquPzyy18eN4PWrWGHHeCww2DbbaFly3C57bawzTbQtGkovAl6/SMr2O7+rZl9Y2Y7ufvnwKHA9KiWH7WGDRvy448/xh1Dss2yZXD22dCuHfz973GnKctQoK+ZDQH2BpZuyv7rJPvDH/7AZZddxq677pqxZRx99NE8/fTTNG7ceL37b7rpJho2bMgVV1yRsWXnpB9/hLffhtGjYexY+OSTX5qqd9gBOnWC3/8eOnSA9u1h++3DFnCWibrjlIuAwWZWB5gN/D7i5aeVu+Pu1Kih09klTa64AubMCV86DRtGvngzewboBmxlZoXAjUBtAHd/FBgGHA3MAlaQ5etwWQYMGJDxZQwbNizjy8h5CxbAK6/A0KGhWP/0U9h9tO++cNNNcOCB0LkzbLFF3EnTJtJK4+5T3b3A3Xd39+PdfUmUy0+HOXPmsNNOO3HWWWfRoUMH+vXrR5cuXdh999258cYbN5h/zJgxHHvssetu9+3blyeeeCLCxJI1hg+H/v3h8svDl00M3P00d2/h7rXdvZW7D3T3R1PFGg8udPcd3L2ju2d192XLly/nmGOOYY899qBDhw48++yzdOvWbV2vbAMHDmTHHXeka9eunHfeefTt2xeAXr16ccEFF7DPPvuw/fbbM2bMGM455xx22WUXevXqte75n3nmGTp27EiHDh24+uqr193ftm1bFi1aBMBtt93GjjvuyAEHHMDnn38e3T+fjYqL4bXX4IQTQhP2+efDtGnwxz/CO+/A99+Hy7/8BQ4+OKeKNSS4a9KNuvTSsG8inTp1gnvv3ehsX3zxBU8++SQ//PADL7zwAh9++CHuTo8ePRg3bhwHHXRQenNJ7vvuOzj3XNhtN+jXL+400YtpfX7zzTfZdttteeONNwBYunQpjzwSTi2fN28e/fr1Y8qUKTRq1IhDDjmEPfbYY93fLlmyhPfff5+hQ4fSo0cP3nvvPQYMGECXLl2YOnUqzZs35+qrr2by5Mk0adKEI444gldeeYXjjz9+3XNMnjyZIUOGMHXqVIqLi+ncuTN77bVXel+HXLB4MTz8cJi+/TYcvPWnP4UzKTp2TNR+5kxSW+4m2G677dhnn3146623eOutt9hzzz3p3LkzM2fO5Isvvog7nmSjvn1h4UL417+yct9aturYsSMjR47k6quv5t1332WLEltkH374Ib/+9a/ZcsstqV27NieffPJ6f3vcccdhZnTs2JGtt96ajh07UqNGDXbbbTfmzJnDxIkT6datG82aNaNWrVqcfvrpjBs3br3nePfddznhhBPYbLPN2HzzzenRo0ck/3fW+PZbuOQSaNMmbDV37hyawQsL4c47Yffd86ZYQzZvYVdiSzhTGjRoAIR92Ndeey3nn39+ufPWqlWLNWvWrLutTiJkA889B888A7fcAnvuGXeaeMS0Pu+4445MmTKFYcOG8ec//5lDDz200n9bt25dAGrUqLHu+trbxcXF6uikOpYtCwdd/uMfYd/06aeH4zs6dIg7Way0hV0NRx55JIMGDVp3NPjcuXNZsGDBevNst912TJ8+nZ9++onvv/+ed955J46oklTz58MFF0CXLnDttXGnyTvz5s1js80244wzzuDKK69kypQp6x7r0qULY8eOZcmSJRQXF/Piiy9W6bm7du3K2LFjWbRoET///DPPPPMMv/71r9eb56CDDuKVV15h5cqVLFu2jNdeey0t/1fWcg8/YNu3Dz9gjz469EfwxBN5X6whm7ewE+CII45gxowZ7LtvGMOkYcOG/N///R/NS5wc37p1a0455RQ6dOhAu3bt2DNft6BkQ+5w3nmht6SnnoJaWh2j9umnn3LllVdSo0YNateuzSOPPLLulKqWLVty3XXX0bVrV7bcckt23nnn9ZrMN6ZFixbcfvvtHHzwwbg7xxxzDD179lxvns6dO/Pb3/6WPfbYg+bNm9OlS5e0/n9Z5b//hT594I03YK+94NVXYe+9406VLGtPTUratNdee3lp06dP3+C+XJLr/5+UMmCAO7jfe2+1ngaY5AlYZyuasnV9XrZsmbu7r1692o899lh/6aWXYsmRDa9VtTz/vPsWW7hvtpn73Xe7r14dd6LYVLQ+q0lcJA5z5oQjow8+GC66KO40Uo6bbrqJTp06rWshK3mEt6TBypVhl9DJJ8NOO8Gnn4ajv9XaVCa9KiJRW7MGevUKR7c+/jio453E+nsye5vLDfPnQ48eMGkSXHkl3HZb6LtbyqWCLRK1++4LPZkNHAjbbRd3GpHoffwxHHdcOL/6lVeg1L59KVvW/bT3qgxllkVy9f+SUmbMCEeDH3ts6Ns4z+lzv3E59xqNGgUHHBBamsaPV7Gugqwq2PXq1WPx4sU59wH21Ji39dRhRm5bvRrOOiv0Ef7YY3nV4UNZcnV9Tqec+24YMQKOOQbatoUPPsjffgc2UVY1ibdq1YrCwsKcHBu2Xr16tGrVKu4Ykkl/+1vYX/f882H4vjyXy+tzOuXMd8Prr8NvfhPGkx45ErbaKu5EWSerCnbt2rVp165d3DFEqm7y5NBH+O9+ByedFHeaRND6nEfeeQdOPBH22CNsZW+5ZdyJslKkBdvM5gDLgJ+BYncviHL5IrEoKgpN4c2bw4MPxp1GJFoffRRG19ppp7BlXWoMcKm8OLawD3b3RTEsVyQef/5z6F5x+HBo0iTuNCLR+eorOOqoUKTffFPFupqyqklcJOuMGwd33x3G7e3ePe40ItFZujQU61WrYPToMH61VEvUR4k78JaZTTaz3hEvWyRay5aFDlLatQsjD4nkizVr4Mwz4csv4eWXYZdd4k6UE6Lewj7A3eeaWXNgpJnNdPd1A8SminhvgDZt2kQcTSTNrrgidEE6blw4lUskX/TrB6+9BvffD6VGKJNNF+kWtrvPTV0uAF4GupZ6vL+7F7h7QbNmzaKMJpJew4dD//6haB9wQNxpqsTMupvZ52Y2y8yuKePxNmY22sw+MrNPzOzoOHJKQr3+Otx0U9jC7ts37jQ5JbKCbWYNzKzR2uvAEcBnUS1fJDLffQfnngu77RbG9M0iZlYTeAg4CtgVOM3Mdi0125+B59x9T+BU4OFoU0pizZsHZ58NnTrBP/+Z950DpVuUTeJbAy9beANrAU+7+5sRLl8kGhdeCAsXhnF9s6+Hqq7ALHefDWBmQ4CewPQS8ziweer6FsC8SBNKMq0d1GblShgyBOrXjztRzomsYKe+APaIankisXjuufBl1a9ftna72BL4psTtQmDvUvPcRDh49CKgAXBYWU+kY1LyzH33hfOsH300nHMtaZdVfYmLJNr8+WFs365d4ZoNdv3mktOAJ9y9FXA08C8z2+C7RMek5JFPPw2f+R49oLdOAMoUFWyRdHCH886DFSvgySehVtZ2cTAXaF3idqvUfSWdCzwH4O7vA/UAdQydr4qL4ZxzQqcoAwZov3UGqWCLpMPAgWGf9e23w847x52mOiYC7c2snZnVIRxUNrTUPP8FDgUws10IBVsjeOSr++8Pg9o88ACoJSWjVLBFquurr+BPf4KDD4aLLoo7TbW4ezHQFxgBzCAcDT7NzG4xsx6p2S4HzjOzj4FngF6uMTLz0+zZoevd446Dk0+OO03Oy9p2O5FEWLMGfv/70Az4+ONQI/t/A7v7MGBYqfv+UuL6dGD/qHNJwrjDH/8INWvCQw+pKTwCKtgi1XHffTB2LAwaBNttF3cakeg8+2w4KvyBB6B1643PL9WW/ZsDInGZPh2uvTY0B/bqFXcakegsXw5XXhlOXbzggrjT5A1tYYtsitWrwxjXDRuGLkjVHCj55I47oLAQnnkmNIlLJFSwRTbFX/8KkyfD88/DNtvEnUYkOl99BXfeCb/7Xdb1k5/t1CQuUlWTJ8Ott4YvrJNOijuNSLSuuCJsVd9xR9xJ8o4KtkhVFBWFpvDmzeHBB+NOIxKt8ePhpZfCsRutWsWdJu+oSVykKv7853Cw2ZtvQpMmcacRiY47XHUVtGgBl10Wd5q8pIItUlnjxsHdd4dzT488Mu40ItF69VV4//1wkOVmm8WdJi+pSVykMpYtC+P8tmsHd90VdxqRaBUXh2bwnXcOHQVJLCLfwjazmsAkYK67Hxv18kU2yeWXw9dfh63shg3jTiMSrccfh5kz4eWXs3lgm6wXxxb2JYQ+ikWyw7Bh8NhjoaMIncYi+aaoCG6+GfbdF3r2jDtNXou0YJtZK+AYYECUyxXZZN99B3/4A3ToALfcEncakegNHAhz50K/fuogKGZRt23cC1wFNCrrQTPrDfQGaNOmTXSpRMpz4YWwcGEYOrNu3bjTiESrqAj+9rfQsnTIIXGnyXuRbWGb2bHAAnefXN487t7f3QvcvaCZxlWVuD37LAwZAjfeGPpMFsk3AwaEreubb9bWdQJE2SS+P9DDzOYAQ4BDzOz/Ily+SOXNnw99+kDXrnDNNXGnEYne2q3rAw8MY71L7CIr2O5+rbu3cve2wKnAKHc/I6rli1Sae9hvvWIFPPWUjoqV/PTYYzBvnrauE0TfRCKlDRwYjgy/7z7Yaae404hEb+3W9UEHQbducaeRlFgKtruPAcbEsWyRCn31FfzpT6EJsG/fuNOIxGPAgLBbaPBgbV0niHo6E1lrzRro1St8QT3+ONTIz9XDzLqb2edmNsvMytyBb2anmNl0M5tmZk9HnVEyqLgY/v532G8/7btOGDWJi6x1772hJ7NBg2C77eJOE4tUT4QPAYcDhcBEMxvq7tNLzNMeuBbY392XmFnzeNJKRjz/fOjV7/77404ipeTnJoRIadOnw3XXwXHHha3s/NUVmOXus919FeGMjtLdW50HPOTuSwDcfUHEGSVT3OHOO0Of4ceq5+ikUcEWWb06jHHdqFE4Mja/99m1BL4pcbswdV9JOwI7mtl7ZjbBzLqX9URm1tvMJpnZpIULF2YorqTV22/D1KmhG9483SWUZGoSF/nrX2HyZHjhBdh667jTZINaQHugG9AKGGdmHd39+5IzuXt/oD9AQUGBR5xRNsWdd8K228Lpp8edRMqgn1CS3yZNCn0kn346/OY3cadJgrlA6xK3W6XuK6kQGOruq939K+A/hAIu2WzKlLCFfeml6oY3oVSwJX+tXBmawrfZBh54IO40STERaG9m7cysDqGTo6Gl5nmFsHWNmW1FaCKfHWFGyYS77oLNN4feveNOIuVQk7jkrz//GWbMgDffhCZN4k6TCO5ebGZ9gRFATWCQu08zs1uASe4+NPXYEWY2HfgZuNLdF8eXWqpt9mx47jm44grYYou400g5VLAlP40dC/fcAxdcAEceGXeaRHH3YcCwUvf9pcR1By5LTZIL7r4bataESy6JO4lUQE3ikn+WLQunbm2/fWgGFMlnCxeGvgfOPDMccCaJpS1syT+XXx46hnj3XWjQIO40IvF66KFwPMcVV8SdRDZCW9iSX954I5xrfeWVsP/+cacRidfy5fDgg9CjB+yyS9xpZCNUsCV/LF4chs3s0AFuuSXuNCLxe/zxsF5cdVXcSaQSImsSN7N6wDigbmq5L7j7jVEtX4QLLwxfTsOH6zxTkeJi+Mc/wiAfam3KClHuw/4JOMTdfzSz2sB4Mxvu7hMizCD56tlnw3TrrdCpU9xpROL3wgswZ04Y9EayQmQFO3UqyI+pm7VTk7orlMybNw/69IG994arr447jUj8Sg7ycdxxcaeRSop0H7aZ1TSzqcACYKS7fxDl8iUPuYf91itXwpNPQi2dGCHCO+/ARx9pkI8sE+k75e4/u3snQv/EXc2sQ8nHNbqPpN2AAWGf9e23w047xZ1GJBnuuANatNAgH1kmlp9WqVF9RgPdS93f390L3L2gWbNmcUSTXPLVV3DZZXDwwdC3b9xpRJJBg3xkrcgKtpk1M7PGqev1gcOBmVEtX/LMmjWhN7MaNeCJJ9TsJ7LWXXeFsd/PPz/uJFJFUe7QawE8aWY1CT8UnnP31yNcvuSTe++FcePCeaZt2sSdRiQZvvoqDPJx+eUa5CMLRXmU+CfAnlEtT/LYtGlw3XWh96azz447jUhyaJCPrKZ2Qsktq1eHMa4bNYL+/cEs7kQiybBoEQwcCGecAS1bxp1GNoHOcZHccttt4aCaF16ArbeOO41Icjz4YDi98cor404im0hb2JI7Jk0KPZmdcQb85jdxpxFJjh9/hAceCJ2kaJCPrKWCLblh5crQFL7NNuGLSUR+MWAAfPcdXHtt3EmkGtQkLrnh+uthxgwYMQIaN447jUhyrFoVBvn49a9h333jTiPVoC1syX5jx4bTuC64AI44Iu40Wc/MupvZ52Y2y8yuqWC+35iZm1lBlPmkigYPhsJCuKbct1KyhAq2ZLdly0IHKdtvHzqEkGpJ9ZPwEHAUsCtwmpntWsZ8jYBLAI0HkGRr1oRuSPfYA448Mu40Uk0q2JLdLrsMvv46DOzRoEHcaXJBV2CWu89291XAEKBnGfP1A+4AiqIMJ1X06qvw+edh61qnOGY9FWzJXm+8EQ6mueoq2H//uNPkipbANyVuF6buW8fMOgOt3f2Nip5Ig/nEzB3+9jfYYQc46aS400gaqGBLdlq8OAyb2bEj3Hxz3GnyhpnVAO4GLt/YvBrMJ2ajR8PEieG8aw0rmxP0Lkp26tMnFO3hwzXiUHrNBVqXuN0qdd9ajYAOwBgLTazbAEPNrIe7T4ospWzc7beHzoPUPW/O0Ba2ZJ8hQ8IABjfeCJ06xZ0m10wE2ptZOzOrA5wKDF37oLsvdfet3L2tu7cFJgAq1kkzaRKMHAl/+hPUqxd3GkkTFWzJLvPmha3rvfeGq6+OO03OcfdioC8wAphBGFVvmpndYmY94k0nlXbzzdCkSTjVUXKGmsQle7iH/dZFReGocO2Xywh3HwYMK3XfX8qZt1sUmaQKJk6E118P3fRuvnncaSSNItvCNrPWZjbazKab2TQz0/huUjUDBoR91nfcATvtFHcakWS6+WbYcku46KK4k0iaRdkkXgxc7u67AvsAF5bVIYNImWbPDvvjDjkELrww7jQiyTRxYjjd8fLLtXWdgyIr2O4+392npK4vI+wf06CssnE//xx6M6tZEx5/HGro0AuRMt10k7auc1gs33xm1hbYk1LdGqqjBSnTvffCu+/CffdBmzZxpxFJpg8/hGHD4IoroFGjuNNIBkResM2sIfAicKm7/1DyMXW0IBuYNi2MxNWzp84nFanIzTdD06bQt2/cSSRDIi3YZlabUKwHu/tLUS5bstDq1WGM60aNoH9/9YUsUp533w1b11deqa3rHBbZeTEWukUaCMxw97ujWq5ksVtvhSlT4MUXoXnzuNOIJJN76E+/ZUu4+OK400gGRXki6/7AmcCnZjY1dd91qXM+RdY3cSLcdhuccQaceGLcaUSS65VXYMKEcNpj/fpxp5EMiqxgu/t4QG2asnErV4am8G22gQceiDuNSHIVF8O118Iuu+gYjzygrqIkea6/HmbOhBEjoHHjuNOIJNfjj4fxrl95RT3/5QGd0CrJMmZMOI2rTx844oi404gk19KlcMMNsN9+0EPdvOcD/SST5Pjhh9BByg47wJ13xp1GJNn69YMFC0K/4TqDIi+oYEtyXHYZfPNNOEWlQYO404gk18yZoSOhc86BgoK400hE1CQuyfDGGzBwYDiPdL/94k4jklzucOmlsNlm8Ne/xp1GIqQtbInf4sVh2MyOHUNvTSJSvtdeCwdk3n23+ifIMyrYEr8+fULRHj4c6taNO41Ici1bFroe3XVXdUGah1SwJV5DhsBzz4VOUjp1ijuNSLJdfz0UFsJ770Ht2nGnkYhpH7bEZ968sHW9zz6ha0URKd+ECfDgg2E8+H33jTuNxEAFW+LhDueeC0VF8OST6vQhQcysu5l9bmazzOyaMh6/zMymm9knZvaOmW0XR868smpVOM6jZUsdaJbHVLAlHo89Bm++CXfcATvuGHcaSTGzmsBDwFHArsBpZrZrqdk+AgrcfXfgBUAnzWfaTTeFoWYfeUSjceUxFWyJ3uzZ4ZzrQw8NzXuSJF2BWe4+291XAUOAniVncPfR7r4idXMC0CrijPll7Fi4/fawhX3ssXGnkRipYEu0fv459GZWs2boB7mGPoIJ0xL4psTtwtR95TkXGF7WA2bW28wmmdmkhQsXpjFiHlmyBM48M/T+d889caeRmEX2bWlmg8xsgZl9FtUyJYHuuSf0ZHb//dC6ddxppBrM7AygALirrMfdvb+7F7h7QbNmzaINlwvcw0GZ8+fD009Dw4ZxJ5KYRbl58wTQPcLlSdJMmxZOS+nZMwyfKUk0Fyj5S6pV6r71mNlhwPVAD3f/KaJs+eXhh8NpjzffDF26xJ1GEiCygu3u44DvolqeJMzq1aFIb7459O+vwQqSayLQ3szamVkd4FRgaMkZzGxP4J+EYr0ghoy5b/z40P3oscfCNRscqC95SufSSDRuvRWmTIEXX1R3ignm7sVm1hcYAdQEBrn7NDO7BZjk7kMJTeANgect/PD6r7trfMd0mTsXTjoJ2rWDf/1Lx3nIOokq2GbWG+gN0KZNm5jTSNpMnBh6MjvzTDjxxLjTyEa4+zBgWKn7/lLi+mGRh8oXy5eHdeTHH+Gdd6Bx47gTSYIk6qebDlLJQStXhqbwFi3CgWYiUrbVq+G3v4VJk2DwYNhtt7gTScIkagtbctB114Wxe996S1sLIuVxh/PPD8PMPvpoODBTpJQoT+t6Bngf2MnMCs3s3KiWLTEZMwbuvTecmnL44XGnEUkmd7j66tAvwY03hsItUobItrDd/bSoliUJ8MMPoYOUX/0K7lTPlSJlcofLLw/9E/TpEwq2SDnUJC6Zcdll8M03oZOUBg3iTiOSPGvWwMUXw0MPhct779XpjlIhFWxJv9deg4EDw/mj++0XdxqR5CkqCn2DDx4ctrDvukvFWjZKBVvSa9EiOO882H33MMKQiKxv4UI44QR4773QP8F116lYS6WoYEv6rO37+LvvYMQIqFs37kQiyfLRR6FTlHnz4Nln4ZRT4k4kWSRR52FLlhsyBJ5/PmxZ77FH3GlEksMdHngA9tknNIePHq1iLVWmgi3pMW9eGNt6n33gqqviTiOSHPPmwfHHhwPLjjgCPv44rCciVaSCLdXnDueeG7YcnnoKamlPiwhr1sAjj8Auu4SOg+65B4YOha22ijuZZCl9s0r19e8Pb74Zmvzat487jUj8xo6FK68M/egfdljovWyHHeJOJVlOW9hSPV9+GU5LOfTQcMCZSD6bMgWOOQa6dYP580OL01tvqVhLWqhgy6b7+efQm1nNmqFbRQ0DKPnIHUaODN3v7rVXOF3r9tvhP/8JI9TplC1JEzWJy6a75x4YPx6eeAJat447jUi05s4NW9CDBsGsWWFEujvuCH2Bb7FF3OkkB6lgy6b57DO4/vpw9OtZZ8WdRiQa8+bB66/Dyy+Hpu41a+DXv4YbbghDY6rvAckgFWypulWrQpHeYgv45z/V5Ce5a8UKmDAhHEQ2fHg4iAygXTu49tpfBrgRiYAKtlTdrbeGHpteegmaN487jUh6rF4NM2bA1Knh8/3hh6FAr14djs8oKIDbboMePWC33fRDVSIXacE2s+7AfUBNYIC73x7l8iUNJk6Ev/41HExzwglxp5EM2Nh6amZ1gaeAvYDFwG/dfU7UOTfJzz+HZu1Zs8IZDl9+Ga7PmgXTp4fWI4D69aFTJ/jTn0KT9/77a7+0xC6ygm1mNYGHgMOBQmCimQ119+lRZZBqWrkyNIW3aAH33x93GsmASq6n5wJL3P1XZnYqcAfw28jD/vQTLF1a9vT997BgAXz77S/T/Plh4I01a355jtq1oW3bcNrV4YfDnnuGQr3jjuHsB5EEiXILuyswy91nA5jZEKAnsGkF+8cf4Yor0pdONu6DD2DmzHCwTePGcaeRzKjMetoTuCl1/QXgQTMzd/dNXurTT8OoUaG3vJLTypUb3ldUBMuXh4JdkVq1YJttwg/M1q2hS5dwu1WrUKB32CHcr8IsWSLKgt0S+KbE7UJg75IzmFlvoDdAmzZtKn62n36CV15Ja0DZiEaNwtGxhx8edxLJnI2upyXncfdiM1sKNAUWlZypSuvzxx+Hg7rq1QtT/fq/XDZpUvb9W2xR8dS4sfoGkJySqIPO3L0/0B+goKCg4l/rTZuGZi4RSaQqrc933BEmESlXlD8/5wIle9dolbpPRJKjMuvpunnMrBawBeHgMxHJoCgL9kSgvZm1M7M6wKnA0AiXLyIbV5n1dChwdur6ScCoau2/FpFKiaxJPLWvqy8wgnC6yCB3nxbV8kVk48pbT83sFmCSuw8FBgL/MrNZwHeEoi4iGRbpPmx3HwYMi3KZIlI1Za2n7v6XEteLgJOjziWS73QIpYiISBawpO56MrOFwNcZXsxWlDoVJSGSmguSmy2puSDz2bZz92YZfP5q0/qsXFWU1GxR5Cp3fU5swY6CmU1y94K4c5SW1FyQ3GxJzQXJzpZLkvo6K1fVJTVb3LnUJC4iIpIFVLBFRESyQL4X7P5xByhHUnNBcrMlNRckO1suSerrrFxVl9RssebK633YIiIi2SLft7BFRESyQl4VbDO7yczmmtnU1HR0OfN1N7PPzWyWmV0TQa67zGymmX1iZi+bWeNy5ptjZp+msk/KcKYKXwMzq2tmz6Ye/8DM2mYyT2qZrc1stJlNN7NpZnZJGfN0M7OlJd7jv5T1XBnKV+H7Y8H9qdfsEzPrHFW2XKT1udJ5Ercup5ab2PU5seuyu+fNRBjD94qNzFMT+BLYHqgDfAzsmuFcRwC1UtfvAO4oZ745wFYRvE4bfQ2APsCjqeunAs9GkKsF0Dl1vRHwnzJydQNej+nzVeH7AxwNDAcM2Af4II6cuTJpfa5UlkSuy6llJXZ9Tuq6nFdb2JXUFZjl7rPdfRUwBOiZyQW6+1vuXpy6OYEwQlKcKvMa9ASeTF1/ATjUzCyTodx9vrtPSV1fBswgjM2cLXoCT3kwAWhsZi3iDpXj8n19TuS6DFm/PseyLudjwe6basIYZGZNyni8JfBNiduFRPshOofwy60sDrxlZpPNrHcGM1TmNVg3T+rLaSnQNIOZ1pNqttsT+KCMh/c1s4/NbLiZ7RZVJjb+/sT92cpFWp8rlvh1GRK5PidyXY508I8omNnbwDZlPHQ98AjQj/Bm9AP+QVihYs3l7q+m5rkeKAYGl/M0B7j7XDNrDow0s5nuPi4ziZPLzBoCLwKXuvsPpR6eQuja78fUPs1XgPYRRdP7k2Zan3NfQtfnRL43OVew3f2wysxnZo8Br5fx0FygdYnbrVL3ZTSXmfUCjgUO9dROkjKeY27qcoGZvUxo7srEh6gyr8HaeQrNrBawBbA4A1nWY2a1CSv3YHd/qfTjJVd4dx9mZg+b2VbunvF+iSvx/mTks5XLtD5XW2LXZUju+pzUdTmvmsRL7WM4AfisjNkmAu3NrJ2Z1SEchDE0w7m6A1cBPdx9RTnzNDCzRmuvEw5sKSt/OlTmNRgKnJ26fhIwqrwvpnRJ7VcbCMxw97vLmWebtfvfzKwr4TMexQ+Jyrw/Q4GzUkeY7gMsdff5mc6Wq7Q+V0oi12VI7vqc6HU5iiPbkjIB/wI+BT5JveAtUvdvCwwrMd/RhCMWvyQ0cWU61yzC/pCpqenR0rkIR3l+nJqmZTpXWa8BcAvhSwigHvB8KvuHwPYRvE4HEJo/PynxWh0N/BH4Y2qevqnX52PCAT/7RfTZKvP9KZXNgIdSr+mnQEEU2XJ10vpc6TyJW5dTy03k+pzkdVk9nYmIiGSBvGoSFxERyVYq2CIiIllABVtERCQLqGCLiIhkARVsERGRLKCCLSIikgVUsEVERLKACraIiEgWUMEWERHJAirYIiIiWUAFW0REJAuoYIuIiGQBFWwREZEsoIItIiKSBWrFHaA8W221lbdt2zbuGCKJN3ny5EXu3izuHCKSWYkt2G3btmXSpElxxxBJPDP7Ou4MIpJ5ahIXERHJAmkp2GY2yMwWmNln5TxuZna/mc0ys0/MrHM6lisiIpIv0rWF/QTQvYLHjwLap6bewCNpWq6IiEheSMs+bHcfZ2ZtK5ilJ/CUuzswwcwam1kLd59fleWsXr2awsJCioqKqhM38erVq0erVq2oXbt23FGy25o1sHIlLF++/rRiBaxeDT//DMXFFU/uv0yw/mV17qusRo2gT59Nfw1EJGdEddBZS+CbErcLU/etV7DNrDdhC5w2bdps8CSFhYU0atSItm3bYmaZSxsjd2fx4sUUFhbSrl27uOMk16pV8NVX8J//hGn2bPjf/8K0YEGYvv8+7pTV16qVCraIAAk7Stzd+wP9AQoKCjbYDCkqKsrpYg1gZjRt2pSFCxfGHSVZVqyAkSNh7Fj4979hypSwlbxWkybQogU0bw6dOsHWW4f7GjTYcNpsM6hdG2rVqniqWRNqpPYarf3MmZV/var3VUYOf9ZFpGqiKthzgdYlbrdK3VdluVys18qH/7FSVq+GV1+FwYNhxIjQvF2vHnTpApdeCh07wo47Qvv2sOWWcacVEcmoqE7rGgqclTpafB9gaVX3XyfF999/z8MPP7zJf9+tWzedX74xRUVw773Qrh2cfDJ8+CGccw68/TYsXQrjxsGdd8KZZ8Lee6tYi0heSMsWtpk9A3QDtjKzQuBGoDaAuz8KDAOOBmYBK4Dfp2O5cVhbsPtov2JmvPoqXHwx/Pe/cPDB8MgjcPTRoXlaRCSPpeso8dM28rgDF6ZjWXG75ppr+PLLL+nUqRMHH3wwn3zyCUuWLGH16tXceuut9OzZkzlz5nDUUUdxwAEH8O9//5uWLVvy6quvUr9+fQCef/55+vTpw/fff8/AgQM58MADY/6vEuCHH+CCC+Dpp0NT98iRcNhhcacSEUmMRB10ViWXXgpTp6b3OTt1Ck2xFbj99tv57LPPmDp1KsXFxaxYsYLNN9+cRYsWsc8++9CjRw8AvvjiC5555hkee+wxTjnlFF588UXOOOMMAIqLi/nwww8ZNmwYN998M2+//XZ6/49s8+WX0KMHfP453HQTXHst1KkTdyoRkUTJ3oKdAO7Oddddx7hx46hRowZz587lf//7HwDt2rWjU6dOAOy1117MmTNn3d+deOKJZd6flz75BA45JJwz/dZb4bqIiGwgewv2RraEozB48GAWLlzI5MmTqV27Nm3btl3XqUvdunXXzVezZk1Wrly57vbax2rWrElxcXG0oZNk2jQ49FCoXx9GjQpHe4uISJk0+EcVNWrUiGXLlgGwdOlSmjdvTu3atRk9ejRff61Bkypt3jw4/PBwPrSKtYjIRmXvFnZMmjZtyv7770+HDh3o0qULM2fOpGPHjhQUFLDzzjvHHS87rFoVTtdauhQmTFCxFhGpBPOq9GscoYKCAi99vvKMGTPYZZddYkoUrZz+Xy++GB54AJ59Fk45Je40Wc/MJrt7Qdw5RCSz1CQu0RozJhTrSy5RsRYRqQIVbInOihXwhz/A9tvDX/8adxoRkayifdgSnX79wjnXo0aFAThERKTSsm4LO6n73NMpJ//Hb76Be+4J/X8ffHDcaUREsk5WFex69eqxePHi3CxoKWvHw65Xr17cUdLrxhvBPWxli4hIlWVVk3irVq0oLCzM+bGi69WrR6tWreKOkT7TpsGTT4buZLfbLu40IiJZKasKdu3atWnXrl3cMaSqbrsNGjSA666LO4mISNbKqiZxyUL//S889xz07g1Nm8adRkQka6lgS2bdd1+4vPjieHOIiGQ5FWzJnKVL4bHH4Le/hTZt4k4jIpLVVLAlc556CpYtg8suizuJiEjWU8GWzBk0CPbaK0wiIlItKtiSGVOmwNSpcO65cScREckJKtiSGYMGQb16cNppcScREckJKtiSfkVFMHgwnHgiNG4cdxoRkZyggi3pN2wYfP899OoVdxIRkZyhgi3p9/zzsNVWGuRDRCSNVLAlvVauhNdeC83htbKq51sRkURTwZb0evNNWL4cTj457iQiIjklLQXbzLqb2edmNsvMrinj8V5mttDMpqamP6RjuZJAa5vDu3WLO4mISE6pdpulmdUEHgIOBwqBiWY21N2nl5r1WXfvW93lSYIVFYXm8NNOU3O4iEiapWMLuyswy91nu/sqYAjQMw3PK9lm7Fj48Uc4/vi4k4iI5Jx0FOyWwDclbhem7ivtN2b2iZm9YGaty3oiM+ttZpPMbNLChQvTEE0i9cYbUL++jg4XEcmAqA46ew1o6+67AyOBJ8uayd37u3uBuxc0a9YsomiSFu7w+utw6KGhaIuISFqlo2DPBUpuMbdK3beOuy92959SNwcAGg0i18ycCV99BcccE3cSEZGclI6CPRFob2btzKwOcCowtOQMZtaixM0ewIw0LFeS5I03wuXRR8ebQ0QkR1X7UF53LzazvsAIoCYwyN2nmdktwCR3HwpcbGY9gGLgO6BXdZcrCfPGG7D77tCmTdxJRERyUlrOvXH3YcCwUvf9pcT1a4Fr07EsSaDly2H8eLj88riTiIjkLPV0JtX37rtQXBwOOBMRkYxQwZbqGzUK6tSB/fePO4mISM5SwZbqe+cd2Hdf2GyzuJOIiOQsFWypnu++g48+gkMOiTuJiEhOU8GW6hkzJnSaov3XIiIZpYIt1TNqFDRoAF26xJ1ERCSnqWBL9bzzDhx0UDjoTEREMkYFWzbdvHmhS1LtvxYRyTgVbNl0o0aFSxVsEZGMU8GWTTdqFDRpAp06xZ1ERCTnqWDLphs3Luy/rqGPkYhIpumbVjbN/Pnw5Zdw4IFxJxERyQsq2LJpxo8PlwccEG8OEZE8oYItm2b8eKhfHzp3jjuJiEheUMGWTTN+POyzD9SuHXcSEZG8oIItVffDDzB1qvZfi4hESAVbqm7CBFizRvuvRUQipIItVffuu1CzZhhSU0REIqGCLVU3fjzsuSc0bBh3EhGRvKGCLVWzalVoEldzuIhIpFSwpWqmTIGiIh1wJiISMRVsqZq1Habsv3+8OURE8owKtlTN+PHwq1/B1lvHnUREJK+oYEvlucO//62taxGRGKSlYJtZdzP73Mxmmdk1ZTxe18yeTT3+gZm1TcdyJWJffAELF6pgi4jEoNoF28xqAg8BRwG7AqeZ2a6lZjsXWOLuvwLuAe6o7nIlBu+9Fy5VsEVEIpeOLeyuwCx3n+3uq4AhQM9S8/QEnkxdfwE41MwsDcuWKL33HjRpAjvvHHcSEZG8k46C3RL4psTtwtR9Zc7j7sXAUqBpGpYtUXrvPdhvP6ihQx9ERKKWqG9eM+ttZpPMbNLChQvjjiMlLV4MM2eqOVxEJCbpKNhzgdYlbrdK3VfmPGZWC9gCWFz6idy9v7sXuHtBs2bN0hBN0ubf/w6XKtgiIrFIR8GeCLQ3s3ZmVgc4FRhaap6hwNmp6ycBo9zd07Bsicp774Wxr7t0iTuJiEheqlXdJ3D3YjPrC4wAagKD3H2amd0CTHL3ocBA4F9mNgv4jlDUJZu89x507gz168edREQkL1W7YAO4+zBgWKn7/lLiehFwcjqWJTH46SeYOBEuvDDuJCIieStRB51JQk2eHIq29l+LiMRGBVs2Th2miIjETgVbNu6992CHHTTgh4hIjFSwpWJrB/w44IC4k4iI5DUVbKmYBvwQEUkEFWypmPZfi4gkggq2VEwDfoiIJIIKtlRMA36IiCSCvoWlfIsWacAPEZGEUMGW8o0bFy4POijeHCIiooItFRgzJvQdrgE/RERip4It5RszJjSH16kTdxIRkbyngi1lW7QIPv0UunWLO4mIiKCCLeVZu/9aBVtEJBFUsKVs2n8tIpIoKthSNu2/FhFJFBVs2ZD2X4uIJI4KtmxI+69FRBJHBVs2pP3XIiKJo4ItGxo9WvuvRUQSRgVb1jdvHnz2GRx6aNxJRESkBBVsWd9bb4XL7t3jzSEiIutRwZb1jRgBW28Nu+8edxIRESlBBVt+8fPPMHIkHHGExr8WEUkYfSvLL6ZMgcWL4cgj404iIiKlVKtgm9mWZjbSzL5IXTYpZ76fzWxqahpanWVKBo0YAWZhC1tERBKlulvY1wDvuHt74J3U7bKsdPdOqalHNZcpmTJsGHTuDM2axZ1ERERKqW7B7gk8mbr+JHB8NZ9P4vLttzBhAvTQ7ykRkSSqbsHe2t3np65/C2xdznz1zGySmU0ws+PLezIz652ab9LChQurGU2q5LXXwB2OPz7uJCIiUoZaG5vBzN4GtinjoetL3nB3NzMv52m2c/e5ZrY9MMrMPnX3L0vP5O79gf4ABQUF5T2XZMKrr0K7dtCxY9xJRESkDBst2O5+WHmPmdn/zKyFu883sxbAgnKeY27qcraZjQH2BDYo2BKTZcvg7behT59w0JmIiCROdZvEhwJnp66fDbxaegYza2JmdVPXtwL2B6ZXc7mSTiNGwE8/Qc+ecScREZFyVLdg3w4cbmZfAIelbmNmBWY2IDXPLsAkM/sYGA3c7u4q2EnywgvQtGkY8ENERBJpo03iFXH3xcAGo0S4+yTgD6nr/wa0YzSpli2DoUOhVy+oVa2Pg4iIZJB6Ost3L78MK1fCGWfEnURERCqggp3vBg8OR4fvu2/cSUREpAIq2Pns22/D0eG/+52ODhcRSTgV7Hw2eDCsWQOnnx53EhER2QgV7Hy1Zg08+mg4MnyXXeJOIyIiG6GCna9GjoRZs0JnKSIikngq2Pnq4YfDqFy/+U3cSUREpBJUsPPR11/D66/DeedB3bpxpxERkUpQwc5Hd90FNWrA+efHnURERCpJBTvfzJ0Ljz0WejZr0ybuNCIiUkkq2Pnmrrvg55/huuviTiIiIlWggp1P5s6Ff/4Tzjor9G4mIiJZQwU7n1x5JbjDDTfEnURERKpIBTtfjBsHzzwDV12lrWsRkSykgp0PVq2Ciy4KB5ldc03caUREZBNoAOR8cMMN8Mkn8OqrsNlmcacREZFNoC3sXPf223DnneGc6x494k4jIiKbSAU7l82eHUbi2nlnuPvuuNOIiEg1qGDnqsWL4eijYfVqeOklNYWLiGQ57cPORYsWwVFHwZw5YVQuDZ8pIpL1VLBzzddfQ/fuoVi/8AIceGDciUREJA3UJJ5LXn8dOneGefNgxAg49ti4E4mISJqoYOeCBQvg3HPhuOPCudYTJ8JBB8WdSkRE0kgFO5stXgw33gg77ghPPRV6MXv//XBbRERyivZhZ5tVq0Jz9+DBMHQorFwJPXvC7beH07dERCQnVatgm9nJwE3ALkBXd59UznzdgfuAmsAAd7+9OsvNK0uXwqefwoQJMHo0vPsuLFsGTZuGMa0vvBB22y3ulCIikmHV3cL+DDgR+Gd5M5hZTeAh4HCgEJhoZkPdfXo1l52d3MO50cuXw5IlYfr++3D53XdQWAj//W+YZs2Cb7755W932gl+97twMNmRR0Lt2rH9GyIiEq1qFWx3nwFgZhXN1hWY5e6zU/MOAXoC1SvYP/wAZ54ZCuDaKYSq+L6N3U7XPKtWwU8/bTitWvXL/GWpUQNatgwHjx10EHToAB07hqO/W7So1ksmIiLZK4p92C2BEpuJFAJ7lzWjmfUGegO0adOm4mddsyZshYY//GUqfbus+yq6XaNG5f5mY/PUqQN164ap5PW1U4MG0KTJhtM220AtHVogIiLr22hlMLO3gW3KeOh6d381nWHcvT/QH6CgoKCCzVCgcWP46KN0Ll5ERCSxNlqw3f2wai5jLtC6xO1WqftERESkkqI4D3si0N7M2plZHeBUYGgEyxUREckZ1SrYZnaCmRUC+wJvmNmI1P3bmtkwAHcvBvoCI4AZwHPuPq16sUVERPJLdY8Sfxl4uYz75wFHl7g9DBhWnWWJiIjkM3VNKiIikgXMKzonOEZmthD4OsOL2QpYlOFlbIqk5oLkZktqLsh8tu3cvVkGn19EEiCxBTsKZjbJ3QvizlFaUnNBcrMlNRckO5uIZA81iYuIiGQBFWwREZEskO8Fu3/cAcqR1FyQ3GxJzQXJziYiWSKv92GLiIhki3zfwhYREckKeVWwzewmM5trZlNT09HlzNfdzD43s1lmdk0Eue4ys5lm9omZvWxmjcuZb46ZfZrKPinDmSp8Dcysrpk9m3r8AzNrm8k8qWW2NrPRZjbdzKaZ2SVlzNPNzJaWeI//kulcJZZd4ftjwf2p1+wTM+scVTYRyX75OI7jPe7+9/IeNLOawEPA4YShQCea2VB3r9743RUbCVzr7sVmdgdwLXB1OfMe7O4ZPd+4kq/BucASd/+VmZ0K3AH8NpO5gGLgcnefYmaNgMlmNrKM9+Zddz82w1nKU9H7cxTQPjXtDTxCOUPNioiUlldb2JXUFZjl7rPdfRUwBOiZyQW6+1upPtcBJhBGNItTZV6DnsCTqesvAIearR0UPDPcfb67T0ldX0bom75lJpeZZj2BpzyYADQ2sxZxhxKR7JCPBbtvqjlykJk1KePxlsA3JW4XEm1ROAcYXs5jDrxlZpPNrHcGM1TmNVg3T+rHxlKgaQYzrSfVBL8n8EEZD+9rZh+b2XAz2y2qTGz8/Yn7syUiWSznmsTN7G1gmzIeup7QBNmP8MXaD/gHoUDGmsvdX03Ncz2h2XdwOU9zgLvPNbPmwEgzm+nu4zKTOLnMrCHwInCpu/9Q6uEphK46f0wdo/AKoQk6Cnp/RCRjcq5gu/thlZnPzB4DXi/joblA6xK3W6Xuy2guM+sFHAsc6uWca+fuc1OXC8zsZULTdSYKQmVeg7XzFJpZLWALYHEGsqzHzGoTivVgd3+p9OMlC7i7DzOzh81sq0zv908tb2PvT0Y+WyKSH/KqSbzU/sITgM/KmG0i0N7M2plZHeBUYGiGc3UHrgJ6uPuKcuZpkDrQCjNrABxB2fnToTKvwVDg7NT1k4BR5f3QSJfUPvKBwAx3v7ucebZZuy/dzLoSPuNR/JCozPszFDgrdbT4PsBSd5+f6Wwikhtybgt7I+40s06EJvE5wPkAZrYtMMDdj04dqd0XGAHUBAa5+7QM53oQqEtoRgWY4O5/LJkL2Bp4OfV4LeBpd38zE2HKew3M7BZgkrsPJRTOf5nZLOA7QlHPtP2BM4FPzWxq6r7rgDap3I8SfjxcYGbFwErg1Ez/kEgp8/0xsz+WyDaMME78LGAF8PsIcolIjlBPZyIiIlkgr5rERUREspUKtoiISBZQwRYREckCKtgiIiJZQAVbREQkC6hgi4iIZAEVbBERkSyggi0iIpIF/h9Vl36z0VNDSwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 576x432 with 3 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "\n",
    "tensor = torch.linspace(-6, 6, 200)\n",
    "tensor = Variable(tensor)\n",
    "\n",
    "np_data = tensor.numpy()\n",
    "\n",
    "y_relu = torch.relu(tensor).data.numpy()\n",
    "y_sigmoid = torch.sigmoid(tensor).data.numpy()\n",
    "y_tanh = torch.tanh(tensor).data.numpy()\n",
    "\n",
    "plt.figure(1, figsize=(8, 6))\n",
    "plt.subplot(221)\n",
    "plt.plot(np_data, y_relu, c=\"red\", label=\"relu\")\n",
    "plt.legend(loc=\"best\")\n",
    "\n",
    "plt.subplot(222)\n",
    "plt.plot(np_data, y_sigmoid, c=\"red\", label=\"sigmoid\")\n",
    "plt.legend(loc=\"best\")\n",
    "\n",
    "plt.subplot(223)\n",
    "plt.plot(np_data, y_tanh, c=\"red\", label=\"tanh\")\n",
    "plt.legend(loc=\"best\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 损失函数"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "1. 均方误差损失函数 `torch.nn.MSELoss(size_average: bool=None, reduce: bool=None, reduction: str='mean')`\n",
    "    - size_average: 基本弃用，参见reduction。默认情况下损失是每个批次中每个损失元素的平均值。False时用每个批次损失之和\n",
    "    - reduce。基本弃用，根据size_average，对每个batch中结果的损失进行平均或求和\n",
    "    - reduction, none, mean and sum.\n",
    "    \n",
    "2. 交叉熵损失函数 `torch.nn.CrossEntropyLoss(weight: Tensor=None, size_average: bool=None, ignore_index:int=-100, reduce: bool = None,  reduction: str):\n",
    "    - weight: 对分类任务中，手动给出个别类别的权重的缩放量\n",
    "    - ignore_index: 指定被忽略且不对输入梯度做贡献的目标值"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# MINIST分类"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "\n",
    "import torchvision.datasets as dsets\n",
    "import torchvision.transforms as transforms\n",
    "\n",
    "from torch.utils.data import DataLoader\n",
    "\n",
    "batch_size = 100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# MINST dataset\n",
    "train_dataset = dsets.MNIST(\n",
    "    root=\"data/pymnist\", \n",
    "    train = True,                         # 选择训练集\n",
    "    transform = transforms.ToTensor(),    # 转换成Tensor变量\n",
    "    download = True                       # 从网上下载图片\n",
    ")\n",
    "\n",
    "\n",
    "test_dataset = dsets.MNIST(\n",
    "    root=\"data/pymnist\",\n",
    "    train = False,\n",
    "    transform = transforms.ToTensor(),    # 转换成Tensor变量\n",
    "    download = True                       # 从网上下载图片\n",
    ")\n",
    "\n",
    "train_loader = torch.utils.data.DataLoader(dataset = train_dataset, batch_size=batch_size, shuffle=True)\n",
    "test_loader = torch.utils.data.DataLoader(dataset = test_dataset, batch_size=batch_size, shuffle=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_data: torch.Size([60000, 28, 28])\n",
      "train_labels: torch.Size([60000])\n",
      "test_data: torch.Size([10000, 28, 28])\n",
      "test_labels: torch.Size([10000])\n",
      "批次大小: 100\n",
      "load_train_data: torch.Size([60000, 28, 28])\n",
      "load_train_labels: torch.Size([60000])\n"
     ]
    }
   ],
   "source": [
    "print(\"train_data:\", train_dataset.train_data.size())\n",
    "print(\"train_labels:\", train_dataset.train_labels.size())\n",
    "print(\"test_data:\", test_dataset.test_data.size())\n",
    "print(\"test_labels:\", test_dataset.test_labels.size())\n",
    "\n",
    "print(\"批次大小:\", train_loader.batch_size)\n",
    "print(\"load_train_data:\", train_loader.dataset.train_data.shape)\n",
    "print(\"load_train_labels:\", train_loader.dataset.train_labels.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Neural_net(\n",
      "  (layer1): Linear(in_features=784, out_features=500, bias=True)\n",
      "  (layer2): Linear(in_features=500, out_features=10, bias=True)\n",
      ")\n"
     ]
    }
   ],
   "source": [
    "# 定义神经网咯\n",
    "import torch.nn as nn\n",
    "\n",
    "input_size = 784     # mnist的像素为28*28\n",
    "hidden_size = 500\n",
    "num_classes = 10     # 输出为10个类别分别对应于0~9\n",
    "\n",
    "class Neural_net(nn.Module):\n",
    "    \n",
    "    def __init__(self, input_num, hidden_size, out_put: str):\n",
    "        u\"\"\"\n",
    "        初始化函数，接受自动以输入特征的维数，隐含层特征维数以及输出层特征维数\n",
    "        \"\"\"\n",
    "        super(Neural_net, self).__init__()\n",
    "        self.layer1 = nn.Linear(input_num, hidden_size)  # 从输入到隐藏层线性处理\n",
    "        self.layer2 = nn.Linear(hidden_size, out_put)    # 从隐藏层到输出层的线性处理\n",
    "        \n",
    "    def forward(self, x):\n",
    "        out = self.layer1(x)     # 输入层到隐藏层的线性计算\n",
    "        out = torch.relu(out)    # 隐藏层激活\n",
    "        out = self.layer2(out)   # 输出层，注意，输出层直接接Loss\n",
    "        return out\n",
    "    \n",
    "net = Neural_net(input_size, hidden_size, num_classes)\n",
    "print(net)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "current epoch = 0\n",
      "current loss = 2.31439\n",
      "current loss = 0.57303\n",
      "current loss = 0.38976\n",
      "current loss = 0.48939\n",
      "current loss = 0.42696\n",
      "current loss = 0.37783\n",
      "current epoch = 1\n",
      "current loss = 0.44937\n",
      "current loss = 0.25238\n",
      "current loss = 0.32870\n",
      "current loss = 0.23103\n",
      "current loss = 0.35310\n",
      "current loss = 0.16711\n",
      "current epoch = 2\n",
      "current loss = 0.22858\n",
      "current loss = 0.21019\n",
      "current loss = 0.27172\n",
      "current loss = 0.25401\n",
      "current loss = 0.17350\n",
      "current loss = 0.17771\n",
      "current epoch = 3\n",
      "current loss = 0.18748\n",
      "current loss = 0.10848\n",
      "current loss = 0.31313\n",
      "current loss = 0.11546\n",
      "current loss = 0.14799\n",
      "current loss = 0.24774\n",
      "current epoch = 4\n",
      "current loss = 0.14501\n",
      "current loss = 0.15295\n",
      "current loss = 0.20693\n",
      "current loss = 0.18635\n",
      "current loss = 0.11104\n",
      "current loss = 0.10318\n",
      "finished training\n"
     ]
    }
   ],
   "source": [
    "# 训练\n",
    "import numpy as np\n",
    "from torch.autograd import Variable\n",
    "\n",
    "\n",
    "learning_rate = 1e-1   # 学习率\n",
    "num_epoches = 5\n",
    "\n",
    "criterion = nn.CrossEntropyLoss()\n",
    "optimizer = torch.optim.SGD(net.parameters(), lr = learning_rate)   # 使用随机梯度下降\n",
    "\n",
    "for epoch in range(num_epoches):\n",
    "    print(\"current epoch = %d\" % epoch)\n",
    "    for i, (images, labels) in enumerate(train_loader):\n",
    "        images = Variable(images.view(-1, 28 * 28))\n",
    "        labels = Variable(labels)\n",
    "        outputs = net(images)                  # 向前传播\n",
    "        loss = criterion(outputs, labels)     # 计算Loss\n",
    "        optimizer.zero_grad()                   # 在反向传播之前清楚网络状态\n",
    "        loss.backward()                        # Loss反向传播\n",
    "        optimizer.step()\n",
    "        \n",
    "        if i % 100 == 0:\n",
    "            print(\"current loss = %.5f\" % loss.item())\n",
    "print(\"finished training\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accureacy = 96.11\n"
     ]
    }
   ],
   "source": [
    "# 预测\n",
    "\n",
    "total = 0\n",
    "correct = 0\n",
    "\n",
    "for images, labels in test_loader:\n",
    "    images = Variable(images.view(-1,  28 * 28))\n",
    "    outputs = net(images)\n",
    "    \n",
    "    _, predicts = torch.max(outputs.data, 1)\n",
    "    \n",
    "    total += int(labels.size(0))\n",
    "    correct += int((predicts == labels).sum())\n",
    "    \n",
    "print(\"Accureacy = %.2f\" % (100 * correct / total))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Cifar10分类"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# MINST dataset\n",
    "train_dataset = dsets.CIFAR10(\n",
    "    root=\"data/pycifar\", \n",
    "    train = True,                         # 选择训练集\n",
    "    transform = transforms.ToTensor(),    # 转换成Tensor变量\n",
    "    download = False                       # 从网上下载图片\n",
    ")\n",
    "\n",
    "\n",
    "test_dataset = dsets.CIFAR10(\n",
    "    root=\"data/pycifar\",\n",
    "    train = False,\n",
    "    transform = transforms.ToTensor(),    # 转换成Tensor变量\n",
    "    download = False                       # 从网上下载图片\n",
    ")\n",
    "\n",
    "train_loader = torch.utils.data.DataLoader(dataset = train_dataset, batch_size=batch_size, shuffle=True)\n",
    "test_loader = torch.utils.data.DataLoader(dataset = test_dataset, batch_size=batch_size, shuffle=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Net(\n",
      "  (layer1): Linear(in_features=3072, out_features=500, bias=True)\n",
      "  (layer2): Linear(in_features=500, out_features=200, bias=True)\n",
      "  (layer3): Linear(in_features=200, out_features=10, bias=True)\n",
      ")\n"
     ]
    }
   ],
   "source": [
    "from torch.autograd import Variable\n",
    "\n",
    "import torch.nn as nn\n",
    "import torch\n",
    "\n",
    "input_size = 3072\n",
    "hidden_size = 500\n",
    "hidden_size2 = 200\n",
    "num_classes = 10\n",
    "num_epochs = 5\n",
    "batch_size = 100\n",
    "learning_rate = 0.001\n",
    "\n",
    "\n",
    "#定义两层神经网络\n",
    "\n",
    "class Net(nn.Module):\n",
    "\n",
    "    def __init__(self,input_size,hidden_size,hidden_size2,num_classes):\n",
    "        super(Net,self).__init__()\n",
    "\n",
    "        self.layer1 = nn.Linear(input_size,hidden_size)\n",
    "        self.layer2 = nn.Linear(hidden_size,hidden_size2)\n",
    "        self.layer3 = nn.Linear(hidden_size2,num_classes)\n",
    "\n",
    "    def forward(self,x):\n",
    "        out = torch.relu(self.layer1(x))\n",
    "        out = torch.relu(self.layer2(out))\n",
    "        out = self.layer3(out)\n",
    "\n",
    "        return out\n",
    "\n",
    "\n",
    "net = Net(input_size,hidden_size,hidden_size2,num_classes)\n",
    "print(net)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 第八章内容 深层网络"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "########################################\n",
    "\n",
    "#第1步：载入数据\n",
    "\n",
    "########################################\n",
    "\n",
    "import torch\n",
    "import torchvision \n",
    "import torchvision.transforms as transforms\n",
    "\n",
    "#使用torchvision可以很方便地下载cifar10数据集，而torchvision下载的数据集为[0, 1]的PILImage格式，我们需要将张量Tensor归一化到[-1, 1]\n",
    "\n",
    "\n",
    "transform = transforms.Compose(\n",
    "    [transforms.ToTensor(), #将PILImage转换为张量\n",
    "    transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))] #将[0, 1]归一化到[-1, 1]\n",
    ")\n",
    "\n",
    "\n",
    "trainset = torchvision.datasets.CIFAR10(\n",
    "    root=\"data/pycifar\", #root表示cifar10的数据存放目录，使用torchvision可直接下载cifar10数据集，也可直接在https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz这里下载（链接来自cifar10官网）\n",
    "    train=True, download=False, \n",
    "    transform=transform #按照上面定义的transform格式转换下载的数据\n",
    ")\n",
    "\n",
    "trainloader = torch.utils.data.DataLoader(\n",
    "    trainset, batch_size=4, #每个batch载入的图片数量，默认为1\n",
    "    shuffle=True, num_workers=2 #载入训练数据所需的子任务数\n",
    ")\n",
    "\n",
    "\n",
    "\n",
    "testset = torchvision.datasets.CIFAR10(\n",
    "    root=\"data/pycifar\", train=False,\n",
    "    download=False, transform=transform\n",
    ")\n",
    "\n",
    "testloader = torch.utils.data.DataLoader(\n",
    "    testset, batch_size=4,\n",
    "    shuffle=False, num_workers=2\n",
    ")\n",
    "\n",
    "\n",
    "cifar10_classes = ('plane', 'car', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship', 'truck')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "deer\n",
      "deer\n",
      "cat\n",
      "frog\n"
     ]
    }
   ],
   "source": [
    "\n",
    "########################################\n",
    "\n",
    "#查看训练数据\n",
    "#备注：该部分代码可以不放入主函数\n",
    "\n",
    "########################################\n",
    "\n",
    "import numpy as np\n",
    "\n",
    "\n",
    "dataiter = iter(trainloader) #随机从训练数据中取一些数据\n",
    "\n",
    "images, labels = dataiter.next() \n",
    "images.shape #(4L, 3L, 32L, 32L)  \n",
    "\n",
    "#我们可以看到images的shape是4*3*32*32，原因是上面载入训练数据trainloader时一个batch里面有4张图片\n",
    "\n",
    "\n",
    "torchvision.utils.save_image(images[1],\"test.jpg\") #我们仅随机保存images中的一张图片看看\n",
    "\n",
    "for i in labels:\n",
    "    print(cifar10_classes[i]) #打印label"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "########################################\n",
    "\n",
    "#第2步：构建卷积神经网络\n",
    "\n",
    "########################################\n",
    "\n",
    "import math\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "\n",
    "\n",
    "cfg = {'VGG16':[64, 64, 'M', 128, 128, 'M', 256, 256, 256, 'M', 512, 512, 512, 'M', 512, 512, 512, 'M']}\n",
    "\n",
    "\n",
    "class VGG(nn.Module):\n",
    "\n",
    "    def __init__(self, net_name):\n",
    "        super(VGG, self).__init__()\n",
    "\n",
    "        #构建网络的卷积层和池化层，最终输出命名features，原因是通常认为经过这些操作的输出为包含图像空间信息的特征层\n",
    "        self.features = self._make_layers(cfg[net_name])\n",
    "\n",
    "        #构建卷积层之后的全连接层以及分类器\n",
    "        self.classifier = nn.Sequential(\n",
    "            nn.Dropout(),\n",
    "            nn.Linear(512, 512), #fc1\n",
    "            nn.ReLU(True),\n",
    "            nn.Dropout(),\n",
    "            nn.Linear(512, 512), #fc2\n",
    "            nn.ReLU(True),\n",
    "            nn.Linear(512, 10), #fc3，最终cifar10的输出是10类\n",
    "        )\n",
    "        \n",
    "        #初始化权重\n",
    "\n",
    "        for m in self.modules():\n",
    "            if isinstance(m, nn.Conv2d):\n",
    "                n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels\n",
    "                m.weight.data.normal_(0, math.sqrt(2. / n))\n",
    "                m.bias.data.zero_()\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = self.features(x) #前向传播的时候先经过卷积层和池化层\n",
    "        x = x.view(x.size(0), -1)\n",
    "        x = self.classifier(x) #再将features（得到网络输出的特征层）的结果拼接到分类器上\n",
    "        return x\n",
    "\n",
    "    \n",
    "\n",
    "    def _make_layers(self, cfg):\n",
    "\n",
    "        layers = []\n",
    "        in_channels = 3\n",
    "        for v in cfg:\n",
    "            if v == 'M':\n",
    "                layers += [nn.MaxPool2d(kernel_size=2, stride=2)]\n",
    "            else:\n",
    "                #conv2d = nn.Conv2d(in_channels, v, kernel_size=3, padding=1)\n",
    "                #layers += [conv2d, nn.ReLU(inplace=True)]\n",
    "                layers += [nn.Conv2d(in_channels, v, kernel_size=3, padding=1),\n",
    "                           nn.BatchNorm2d(v),\n",
    "                           nn.ReLU(inplace=True)]\n",
    "                in_channels = v\n",
    "        return nn.Sequential(*layers)\n",
    "\n",
    "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
    "\n",
    "net = VGG('VGG16')\n",
    "net = net.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "########################################\n",
    "\n",
    "#第3步：定义损失函数和优化方法\n",
    "\n",
    "########################################\n",
    "\n",
    "import torch.optim as optim\n",
    "\n",
    "#x = torch.randn(2,3,32,32)\n",
    "#y = net(x)\n",
    "#print(y.size())\n",
    "\n",
    "criterion = nn.CrossEntropyLoss().cuda() #定义损失函数：交叉熵\n",
    "optimizer = optim.SGD(net.parameters(), lr=0.001, momentum=0.9) #定义优化方法：随机梯度下降"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1,  2000] loss: 1.380\n",
      "[1,  4000] loss: 1.322\n",
      "[1,  6000] loss: 1.247\n",
      "[1,  8000] loss: 1.210\n",
      "[1, 10000] loss: 1.149\n",
      "[1, 12000] loss: 1.115\n",
      "Saving epoch 1 model ...\n",
      "[2,  2000] loss: 1.034\n",
      "[2,  4000] loss: 1.024\n",
      "[2,  6000] loss: 0.999\n",
      "[2,  8000] loss: 0.963\n",
      "[2, 10000] loss: 0.951\n",
      "[2, 12000] loss: 0.958\n",
      "Saving epoch 2 model ...\n",
      "[3,  2000] loss: 0.842\n",
      "[3,  4000] loss: 0.834\n",
      "[3,  6000] loss: 0.851\n",
      "[3,  8000] loss: 0.801\n",
      "[3, 10000] loss: 0.816\n",
      "[3, 12000] loss: 0.798\n",
      "Saving epoch 3 model ...\n",
      "[4,  2000] loss: 0.691\n",
      "[4,  4000] loss: 0.694\n",
      "[4,  6000] loss: 0.707\n",
      "[4,  8000] loss: 0.699\n",
      "[4, 10000] loss: 0.706\n",
      "[4, 12000] loss: 0.660\n",
      "Saving epoch 4 model ...\n",
      "[5,  2000] loss: 0.567\n",
      "[5,  4000] loss: 0.589\n",
      "[5,  6000] loss: 0.590\n",
      "[5,  8000] loss: 0.589\n",
      "[5, 10000] loss: 0.595\n",
      "[5, 12000] loss: 0.587\n",
      "Saving epoch 5 model ...\n",
      "Finished Training\n"
     ]
    }
   ],
   "source": [
    "########################################\n",
    "\n",
    "#第4步：卷积神经网络的训练\n",
    "\n",
    "########################################\n",
    "import os\n",
    "\n",
    "for epoch in range(5):  #训练数据集的迭代次数，这里cifar10数据集将迭代2次\n",
    "    train_loss = 0.0\n",
    "\n",
    "    for batch_idx, data in enumerate(trainloader):\n",
    "        #初始化\n",
    "        inputs, labels = data[0].to(device), data[1].to(device)\n",
    "        optimizer.zero_grad() #先将梯度置为0\n",
    "\n",
    "        #优化过程\n",
    "        outputs = net(inputs) #将数据输入到网络，得到第一轮网络前向传播的预测结果outputs\n",
    "        loss = criterion(outputs, labels) #预测结果outputs和labels通过之前定义的交叉熵计算损失\n",
    "        loss.backward() #误差反向传播\n",
    "        optimizer.step() #随机梯度下降方法（之前定义）优化权重\n",
    "\n",
    "        #查看网络训练状态\n",
    "        train_loss += loss.item()\n",
    "        if batch_idx % 2000 == 1999: #每迭代2000个batch打印看一次当前网络收敛情况\n",
    "            print('[%d, %5d] loss: %.3f' % (epoch + 1, batch_idx + 1, train_loss / 2000))\n",
    "            train_loss = 0.0\n",
    "\n",
    "    print('Saving epoch %d model ...' % (epoch + 1))\n",
    "\n",
    "    state = {\n",
    "        'net': net.state_dict(),\n",
    "        'epoch': epoch + 1,\n",
    "    }\n",
    "\n",
    "    if not os.path.isdir('checkpoint'):\n",
    "        os.mkdir('checkpoint')\n",
    "    torch.save(state, './checkpoint/cifar10_epoch_%d.ckpt' % (epoch + 1))\n",
    "\n",
    "print('Finished Training')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy of the network on the 10000 test images: 77 %\n"
     ]
    }
   ],
   "source": [
    "########################################\n",
    "\n",
    "#第5步：批量计算整个测试集预测效果\n",
    "\n",
    "########################################\n",
    "\n",
    "correct = 0\n",
    "total = 0\n",
    "\n",
    "with torch.no_grad():\n",
    "    for data in testloader:\n",
    "        images, labels = data[0].to(device), data[1].to(device)\n",
    "        outputs = net(images)\n",
    "        _, predicted = torch.max(outputs.data, 1)\n",
    "        total += labels.size(0)\n",
    "        correct += (predicted == labels).sum().item() #当标记的label种类和预测的种类一致时认为正确，并计数\n",
    "\n",
    "print('Accuracy of the network on the 10000 test images: %d %%' % (100 * correct / total))\n",
    "\n",
    "#结果打印：Accuracy of the network on the 10000 test images: 73 %"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
